import scrapy
from scrapy.http import Request
import re
import json
from ..items import JDItem

from ..items import DoubanItem


class DoubanSpider(scrapy.Spider):
    name = 'douban'
    # start_urls = [
    #     'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100009464821&score=0&sortType=5&page=1&pageSize=10&isShadowSku=0&rid=0&fold=1']
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36'
    }
    pages = 10

    def start_requests(self):
        for i in range(10):
            url = "https://movie.douban.com/top250?start={}&filter=".format(i * 25)
            yield Request(url, headers=self.headers, callback=self.my_parse)

    def my_parse(self, response: scrapy.http.Response):
        # response 中保存了URL返回的HTML文件内容
        print(response.text)

        # 解析页面 获取对于标签中的内容
        '''
            1.电影名
            2.导演
            3.主演
            4.上映年份
            5.拍摄国家
            6.电影类型
            7.评分
            8.评价人数
            9.主题
        '''

        lis = response.xpath('//*[@id="content"]/div/div[1]/ol/li')

        for li in lis:
            douban = DoubanItem()
            movie_name = li.xpath("./div/div[2]/div[1]/a/span[1]/text()").get()
            p_text = li.xpath(".//div[@class='bd']/p[1]/text()")[0].get()
            # 对内容进行切分
            dy_name = p_text.split("主演:")[0].replace('导演:', '').strip()
            zy_name = ""
            try:
                zy_name = p_text.split("主演:")[1].replace("/...", '').strip()
            except IndexError as e:
                print(e, "主演信息未获取到...")
            p_text_next = li.xpath(".//div[@class='bd']/p[1]/text()")[1].get()
            p_next_list = p_text_next.split('\xa0/\xa0')
            year = p_next_list[0].strip()
            country = p_next_list[1].strip()
            movie_type = p_next_list[2].strip()

            score = li.xpath(".//div[@class='bd']/div[@class='star']/span[@class='rating_num']/text()").get()
            people_num = li.xpath(".//div[@class='bd']/div[@class='star']/span[4]/text()").get()
            topic_desc = li.xpath(".//div[@class='bd']/p[@class='quote']/span/text()").get()

            douban['movie_name'] = movie_name
            douban['dy_name'] = dy_name
            douban['zy_name'] = zy_name
            douban['year'] = year
            douban['country'] = country
            douban['score'] = score
            douban['people_num'] = people_num
            douban['topic_desc'] = topic_desc
            douban['movie_type'] = movie_type

            yield douban

            # print("{},{},{},{},{},{},{},{},{}".format(
            #     movie_name,
            #     dy_name,
            #     zy_name,
            #     year,
            #     country,
            #     movie_type,
            #     score,
            #     people_num,
            #     topic_desc,
            # ))
